﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using TextMining.Service;
using DocumentMining.Topic;
using System.Net;
using HtmlAgilityPack;
using System.Text;
using System.IO;
using TextMining.Helper;
using Microsoft.Office.Interop.Word;
using System.Text.RegularExpressions;
using System.Web.Services;

namespace TextMining
{
    public partial class Clustering : System.Web.UI.Page
    {
        #region Properties

        private DocumentCollection docCollection;
        private Website _myWebsite;

        public Website MyWebsite { get; set; }

        private MyDocumentCollection _myDocument;
        public MyDocumentCollection MyDocument
        {
            get { return _myDocument; }
            set { _myDocument = value; }
        }


        private int iTotalDoc;

        #endregion
        protected void Page_Load(object sender, EventArgs e)
        {
            if (docCollection == null || !IsPostBack)
                docCollection = new DocumentCollection() { DocumentList = new List<string>() };
            MyDocument = new MyDocumentCollection();
            MyDocument.KeyInDoc = new List<string>();
            MyDocument.TitleInDosc = new List<string>();
            iTotalDoc = 0;
        }


        #region Classification for Web

        protected void btnAddMoreLink_Click(object sender, System.EventArgs e)
        {
            MyWebsite = new Website();
            //MyWebsite.Id = System.Guid.NewGuid().ToString();
            MyWebsite.Url = txtWeb1.Text;

            string html;
            // obtain some arbitrary html....
            WebClient client = new WebClient();
            using (client = new WebClient())
            {
                html = client.DownloadString(txtWeb1.Text);
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            StringBuilder sb = new StringBuilder();
            bool isAlreadyInsertNewLine = false;
            foreach (HtmlTextNode node in doc.DocumentNode.SelectNodes("//text()"))
            {

                Topic oTopic = new Topic();
                foreach (string item in oTopic.CharHTML)
                {
                    node.Text = Regex.Replace(node.Text, item, string.Empty);
                }
                sb.AppendLine(node.Text);
                MyStatament myStatement = new MyStatament();
                myStatement.Statament = node.Text.Replace("\n", string.Empty).Replace("\r", string.Empty);
                MyWebsite.Statement.Add(myStatement);
            }

            if (MyWebsite.Statement.Count > 0)
            {
                List<MyStatament> lstTmp = new List<MyStatament>();
                //MyWebsite.RawData = new List<string>();

                foreach (MyStatament item in MyWebsite.Statement)
                {
                    if (!string.IsNullOrEmpty(item.Statament))
                    {
                        lstTmp.Add(item);
                        string[] subitem = item.Statament.Split(' ');
                        for (int i = 0; i < subitem.Length; i++)
                        {
                            if (subitem[i] != "")
                            {
                                MyWebsite.RawData.Add(subitem[i]);
                            }
                        }
                    }
                }
                MyWebsite.Statement = lstTmp;
            }

            MyWebsite.CreateWebsiteInfo();


        }

        protected void btnClassification_Click(object sender, System.EventArgs e)
        {
            MyWebsite = new Website();
            //MyWebsite.Id = System.Guid.NewGuid().ToString();
            MyWebsite.Url = txtWeb1.Text;

            string html;
            // obtain some arbitrary html....
            WebClient client = new WebClient();
            using (client = new WebClient())
            {
                html = client.DownloadString(txtWeb1.Text);
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            StringBuilder sb = new StringBuilder();
            bool isAlreadyInsertNewLine = false;
            foreach (HtmlTextNode node in doc.DocumentNode.SelectNodes("//text()"))
            {

                Topic oTopic = new Topic();
                foreach (string item in oTopic.CharHTML)
                {
                    node.Text = Regex.Replace(node.Text, item, string.Empty);
                }
                sb.AppendLine(node.Text);
                MyStatament myStatement = new MyStatament();
                myStatement.Statament = node.Text.Replace("\n", string.Empty).Replace("\r", string.Empty);
                if (!string.IsNullOrEmpty(myStatement.Statament) && myStatement.Statament != "")
                    MyWebsite.Statement.Add(myStatement);
            }

            if (MyWebsite.Statement.Count > 0)
            {
                List<MyStatament> lstTmp = new List<MyStatament>();
                //MyWebsite.RawData = new List<string>();

                foreach (MyStatament item in MyWebsite.Statement)
                {
                    if (!string.IsNullOrEmpty(item.Statament))
                    {
                        lstTmp.Add(item);
                        string[] subitem = item.Statament.Split(' ');
                        for (int i = 0; i < subitem.Length; i++)
                        {
                            if (subitem[i] != "" && !string.IsNullOrEmpty(subitem[i]))
                            {
                                MyWebsite.RawData.Add(subitem[i]);
                            }
                        }
                    }
                }
                MyWebsite.Statement = lstTmp;
            }

            MyWebsite.CreateWebsiteInfo();
            ShowClassification();

        }

        public void ShowClassification()
        {
            List<MyStatament> lstResul = new List<MyStatament>();
            lstResul = MyWebsite.GetListKeyWords();
            StringBuilder sb = new StringBuilder();
            if (lstResul.Count > 0)
            {



                //Website
                //sb.Append("<td rowspan='4'>");
                //sb.AppendFormat("<a href='{0}'>{0}</a>",txtWeb1.Text);
                //sb.Append("</td>");

                //Keywords
                int iCount = 3;
                if (lstResul.Count > 3)
                {
                    for (int k = 0; k < iCount; k++)
                    {
                        sb.Append("<tr>");
                        sb.Append("<td>" + Convert.ToInt32(k + 1).ToString() + ". ");
                        sb.AppendFormat("<a href='https://www.google.com.vn/search?q={0}' target='_bank'>{0}</a>", lstResul[k].Statament);
                        sb.Append("</td>");

                        if (k == 0)
                        {
                            sb.Append("<td>");
                            sb.AppendFormat("{0}", FindClass());
                            sb.Append("</td>");
                        }

                        sb.Append("</tr>");
                    }
                }
                else
                {
                    int index = 1;
                    foreach (MyStatament item in lstResul)
                    {
                        sb.Append("<tr>");
                        sb.Append("<td>" + index.ToString() + ". ");
                        sb.AppendFormat("<a href='https://www.google.com.vn/search?q={0}' target='_bank'>{0}</a>", item.Statament);
                        sb.Append("</td>");
                        index += 1;
                        sb.Append("</tr>");
                    }
                }

                //Classs

            }
            // litClass.Text = sb.ToString();
        }

        public string FindClass()
        {
            string sClass = "Comming soon ...";
            string sFolder = Server.MapPath("Cat\\");
            DirectoryInfo di = new DirectoryInfo(sFolder);
            FileInfo[] aFileInfos = di.GetFiles("*.txt");
            List<string> lstFile = new List<string>();
            SortedList<string, double> sortedListFiles = new SortedList<string, double>();

            if (aFileInfos.Length > 0)
            {

                foreach (var item in aFileInfos)
                {
                    try
                    {
                        double dWeight = 0;
                        using (StreamReader sr = new StreamReader(item.FullName))
                        {
                            string sContent = sr.ReadToEnd();
                            if (!string.IsNullOrEmpty(sContent) && sContent != "")
                            {
                                string[] arrCat = sContent.Split(',');
                                if (arrCat.Length > 1)
                                {
                                    foreach (var sWords in MyWebsite.ListWords)
                                    {
                                        if (!string.IsNullOrEmpty(sWords.Word) && sWords.Word != "")
                                        {
                                            if (sContent.Contains(sWords.Word))
                                            {
                                                dWeight += Convert.ToDouble(MyWebsite.HashWords[sWords.Word]);
                                            }
                                        }

                                    }
                                }
                            }
                            sortedListFiles.Add(item.Name.Remove(item.Name.IndexOf(".")), dWeight);
                        }
                    }
                    catch (Exception ex)
                    {
                        Logger.FrameworkLogger.Error("Read file " + ex.ToString());
                    }
                }
            }

            if (sortedListFiles.Count > 0)
            {
                double dMax = 0;
                foreach (var item in sortedListFiles)
                {
                    if (item.Value > dMax)
                    {
                        sClass = item.Key;
                        dMax = item.Value;
                    }
                }
            }

            return sClass;
        }

        #endregion

        #region Clustering for document

        protected void btnAddMore_Click(object sender, System.EventArgs e)
        {
            //string sDoc1 = txtDoc1.Text;
            //int iNumCluster = Convert.ToInt32(txtNumCluster.Text);

            if (docCollection == null)
                docCollection = new DocumentCollection() { DocumentList = new List<string>() };

            if (string.IsNullOrEmpty(txtDoc1.Text) && flDoc1.HasFile == true)
            {
                ReadFile(flDoc1, txtDoc1);
            }


            if (string.IsNullOrEmpty(txtDoc2.Text) && flDoc2.HasFile == true)
            {
                ReadFile(flDoc2, txtDoc2);
            }

            if (string.IsNullOrEmpty(txtDoc3.Text) && flDoc3.HasFile == true)
            {
                ReadFile(flDoc3, txtDoc3);
            }

            if (string.IsNullOrEmpty(txtDoc4.Text) && flDoc4.HasFile == true)
            {
                ReadFile(flDoc4, txtDoc4);
            }

            if (!string.IsNullOrEmpty(txtDoc1.Text))
            {
                docCollection.DocumentList.Add(txtDoc1.Text);
            }
            if (!string.IsNullOrEmpty(txtDoc2.Text))
            {
                docCollection.DocumentList.Add(txtDoc2.Text);
            }
            if (!string.IsNullOrEmpty(txtDoc3.Text))
            {
                docCollection.DocumentList.Add(txtDoc3.Text);
            }
            if (!string.IsNullOrEmpty(txtDoc4.Text))
            {
                docCollection.DocumentList.Add(txtDoc4.Text);
            }

            txtDoc1.Text = string.Empty;
            txtDoc2.Text = string.Empty;
            txtDoc3.Text = string.Empty;
            txtDoc4.Text = string.Empty;



        }

        public void ReadToWords(string sMyFile)
        {
            if (sMyFile.Length > 0)
            {
                try
                {
                    string[] arrWords = sMyFile.Split(' ');

                    Website myCurrSite = new Website();
                    myCurrSite.ListWords = new List<MyWords>();
                    foreach (string item in arrWords)
                    {
                        if (!string.IsNullOrEmpty(item))
                        {
                            MyWebsite.RawData.Add(item);
                            MyWords myWords = new MyWords();
                            myWords.Word = item;
                            myCurrSite.ListWords.Add(myWords);
                        }
                    }


                    if (arrWords.Length > 0)
                    {

                        if (MyWebsite.RawData != null)
                        {
                            MyWebsite.ListWords = new List<MyWords>();


                            for (int i = 0; i < MyWebsite.RawData.Count; i++)
                            {
                                MyWords myWords = new MyWords();
                                if (i == 0)
                                {
                                    myWords.Word = MyWebsite.RawData[0].ToString();
                                    myWords.Count = 1;
                                    myWords.Next = new List<MyWords>();
                                    MyWords nextWord = new MyWords();
                                    nextWord.Word = MyWebsite.RawData[1].ToString();
                                    myWords.Next.Add(nextWord);
                                    myWords.Previous = null;
                                }
                                else
                                {
                                    int iPos = MyWebsite.FindWordInList(MyWebsite.RawData[i].ToString(), MyWebsite.ListWords);
                                    if (iPos > -1)  // Exist word  in list
                                    {
                                        MyWords exsitWord = new MyWords();
                                        exsitWord = MyWebsite.ListWords[iPos];
                                        exsitWord.Count += 1;
                                        MyWords prevtWord = new MyWords();
                                        prevtWord.Word = MyWebsite.RawData[i - 1].ToString();
                                        exsitWord.Previous = new List<MyWords>();
                                        exsitWord.Previous.Add(prevtWord);

                                        if (i == MyWebsite.RawData.Count - 1) // the last word
                                        {

                                            exsitWord.Next = null;
                                        }
                                        else // Not last word
                                        {
                                            exsitWord.Next = new List<MyWords>();
                                            MyWords nextWord = new MyWords();
                                            nextWord.Word = MyWebsite.RawData[i + 1].ToString();
                                            exsitWord.Next.Add(nextWord);
                                        }
                                        MyWebsite.ListWords[iPos] = exsitWord;
                                    }
                                    else  //No exist word
                                    {
                                        myWords.Word = MyWebsite.RawData[i].ToString();
                                        myWords.Count = 1;
                                        myWords.Next = new List<MyWords>();
                                        myWords.Previous = new List<MyWords>();
                                        MyWords prevWord = new MyWords();
                                        prevWord.Word = MyWebsite.RawData[i - 1].ToString();
                                        myWords.Previous.Add(prevWord);
                                        if (i == MyWebsite.RawData.Count - 1) //The last word
                                        {
                                            myWords.Next = null;
                                        }
                                        else //Not last word
                                        {
                                            MyWords nextWord = new MyWords();
                                            nextWord.Word = MyWebsite.RawData[i + 1].ToString();
                                            myWords.Next.Add(nextWord);
                                        }
                                    }
                                }
                                MyWebsite.ListWords.Add(myWords);

                            }


                            MyWebsite.SortListWords("desc");
                            MyWebsite.CreateHashtableWords();

                            #region Calculate Max words

                            myCurrSite.ListWords.Sort(
                                delegate(MyWords p1, MyWords p2)
                                {
                                    return p1.Count.CompareTo(p2.Count);
                                }
                            );
                            myCurrSite.ListWords.Reverse();


                            if (myCurrSite.ListWords.Count > 0)
                            {
                                string sKey = string.Empty;
                                for (int i = 0; i < 3; i++)
                                {
                                    sKey = myCurrSite.ListWords[i].Word;
                                    if (myCurrSite.ListWords[i].Next != null)
                                        foreach (MyWords item in MyWebsite.ListWords)
                                        {
                                            if (string.Equals(item.Word, sKey))
                                            {
                                                if (item.Next != null)
                                                    sKey = string.Format(sKey + " {0}", item.Next[0].Word);
                                                if (item.Previous != null)
                                                    sKey = string.Format("{0} " + sKey, item.Previous[0].Word);
                                            }
                                        }
                                }
                                MyDocument.KeyInDoc.Add(sKey);
                            }
                            myCurrSite.ListWords = null;
                            #endregion
                        }
                    }
                }
                catch (Exception ex)
                {
                    Logger.FrameworkLogger.Error("Error when parse data: ", ex);
                }
            }

        }



        protected void btnRemoveAll_Click(object sender, System.EventArgs e)
        {
            docCollection = new DocumentCollection();
            docCollection.DocumentList = new List<string>();

            txtDoc1.Text = string.Empty;
            txtDoc2.Text = string.Empty;
            txtDoc3.Text = string.Empty;
            txtDoc4.Text = string.Empty;

        }
        protected void btnExe_Click(object sender, System.EventArgs e)
        {
            try
            {


                MyWebsite = new Website();
                if (string.IsNullOrEmpty(txtDoc1.Text) && flDoc1.HasFile == true)
                {
                    ReadFile(flDoc1, txtDoc1);


                }



                if (string.IsNullOrEmpty(txtDoc2.Text) && flDoc2.HasFile == true)
                {
                    ReadFile(flDoc2, txtDoc2);

                }

                if (string.IsNullOrEmpty(txtDoc3.Text) && flDoc3.HasFile == true)
                {
                    ReadFile(flDoc3, txtDoc3);

                }

                if (string.IsNullOrEmpty(txtDoc4.Text) && flDoc4.HasFile == true)
                {
                    ReadFile(flDoc4, txtDoc4);

                }

                docCollection.DocumentList.Add(txtDoc1.Text);
                docCollection.DocumentList.Add(txtDoc2.Text);
                docCollection.DocumentList.Add(txtDoc3.Text);
                docCollection.DocumentList.Add(txtDoc4.Text);
                foreach (var itemDoc in docCollection.DocumentList)
                {
                    if (!string.IsNullOrEmpty(itemDoc))
                        iTotalDoc += 1;
                }

                if (int.Parse(txtNumCluster.Text) > 0 && int.Parse(txtNumCluster.Text) < iTotalDoc && iTotalDoc >0)
                {

                    //if (!string.IsNullOrEmpty(txtDoc1.Text))
                    //    ReadToWords(txtDoc1.Text);
                    //if (!string.IsNullOrEmpty(txtDoc2.Text))
                    //    ReadToWords(txtDoc2.Text);
                    //if (!string.IsNullOrEmpty(txtDoc3.Text))
                    //    ReadToWords(txtDoc3.Text);
                    //if (!string.IsNullOrEmpty(txtDoc4.Text))
                    //    ReadToWords(txtDoc4.Text);



                    List<DocumentVector> vSpace = VectorSpaceModel.ProcessDocumentCollection(docCollection);

                    int totalIteration = 0;
                    List<Centroid> resultSet = DocumnetClustering.PrepareDocumentCluster(int.Parse(txtNumCluster.Text), vSpace, ref  totalIteration);
                    string msg = string.Empty;


                    StringBuilder sb = new StringBuilder();
                    int iGroup = 0;
                    foreach (Centroid c in resultSet)
                    {
                        iGroup += 1;
                        int iDocInClustering = c.GroupedDocument.Count;
                        int iIndex = 0;
                        foreach (DocumentVector document in c.GroupedDocument)
                        {

                            if (iIndex == 0)
                            {
                                sb.Append("<tr class='tr-first'>");
                                sb.AppendFormat("<td style='vertical-align:top;' rowspan='" + c.GroupedDocument.Count.ToString() + "'>" + iGroup.ToString() + "</td>");
                            }
                            else
                                sb.Append("<tr>");
                            sb.Append("<td><i style='font-size:12px;'>&ldquo;");
                            int iDocLen = document.Content.Length;
                            if (iDocLen > 201)
                                sb.Append(document.Content.Substring(0, 200) + "...&rdquo;</i></td>");
                            else
                                sb.Append(document.Content + "&rdquo;</i></td>");


                            //if (document.KeyInDoc.Count > 0)
                            //{
                            //    sb.Append("<td>");
                            //    //foreach (string item in MyDocument.KeyInDoc)
                            //    //{
                            //    sb.Append(document.KeyInDoc[0]);
                            //    //}
                            //    //if(MyDocument.KeyInDoc[iIndex] != null)
                            //    //    sb.Append(MyDocument.KeyInDoc[iIndex]);
                            //    sb.Append("</td>");
                            //}
                            //else
                            sb.Append("<td>Comming soon ....</td>");



                            sb.Append("</tr>");
                            iIndex += 1;
                        }
                    }
                    litResult.Text = sb.ToString();
                    panResult.Visible = true;
                    litResult.Text = string.Empty;
                }
                else
                {
                    if (iTotalDoc == 0)
                        lbResult.Text = "Xin vui lòng nhập nội dung cần gom nhóm";
                    else
                        lbResult.Text = "Số nhóm phải lớn hơn 0 và nhỏ hơn hoặc bằng " + iTotalDoc.ToString();
                }
            }
            catch (Exception ex)
            {
                Logger.FrameworkLogger.Error("btnExe_Click ", ex);
            }



        }
        private void ReadFile(FileUpload sFileName, TextBox txt)
        {
            StringBuilder sb = new StringBuilder();

            try
            {
                string fn = System.IO.Path.GetFileName(sFileName.PostedFile.FileName);
                string AdsFile = Guid.NewGuid().ToString() + Path.GetExtension(sFileName.FileName);
                string SaveLocation = Server.MapPath("Upload") + "\\" + AdsFile;
                sFileName.PostedFile.SaveAs(SaveLocation);
                if (!string.IsNullOrEmpty(AdsFile))
                {


                    if (Path.GetExtension(SaveLocation) == ".txt")  // TXT file
                    {
                        try
                        {
                            using (StreamReader sr = new StreamReader(SaveLocation))
                            {
                                String line = sr.ReadToEnd();
                                sb.Append(line);
                            }
                        }
                        catch (Exception ex)
                        {
                            Logger.FrameworkLogger.Error("Read file " + ex.ToString());
                        }

                    }
                    else   // Microsoft word
                    {
                        object fileName = SaveLocation;
                        object readOnly = true;
                        object isVisible = true;
                        object missing = System.Reflection.Missing.Value;
                        Microsoft.Office.Interop.Word.Application oWordApp = new Microsoft.Office.Interop.Word.Application();
                        Microsoft.Office.Interop.Word.Document oWordDoc = new Document();
                        try
                        {
                            oWordApp.Visible = false;
                            oWordDoc = oWordApp.Documents.Open(ref fileName,
                                              ref missing, ref readOnly,
                                              ref missing, ref missing, ref missing,
                                              ref missing, ref missing, ref missing,
                                              ref missing, ref missing, ref isVisible,
                                              ref missing, ref missing, ref missing);

                            oWordDoc.Activate();
                        }
                        catch (Exception ex)
                        {
                            Logger.FrameworkLogger.Error("Read file ", ex);
                        }
                        finally
                        {
                            //oWordApp.Visible = true;
                        }
                        object startPosition = 0;
                        object endPosition = (object)oWordDoc.Characters.Count;
                        Range range = oWordDoc.Range(ref startPosition, ref endPosition);
                        string text = range.Text;
                        if (text.Length <= 0)
                            text = "\"Có thể file của bạn đã bị lỗi, xin vui lòng mở file và nhập trực tiếp vào ô văn bản trên màn hình\"";
                        sb.Append(text);
                    }
                }

            }
            catch (Exception ex)
            {
                Logger.FrameworkLogger.Error("Read file ", ex);
            }



            txt.Text = sb.ToString();
        }

        [WebMethod]
        public static string GetContentFromUrl(string sUrl)
        {

            string html;
            // obtain some arbitrary html....
            WebClient client = new WebClient();
            using (client = new WebClient())
            {
                html = client.DownloadString(sUrl);
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);

            var text = doc.DocumentNode.Descendants()
              .Where(x => x.NodeType == HtmlNodeType.Text && x.InnerText.Trim().Length > 0)
              .Select(x => x.InnerText.Trim()).ToList();

            StringBuilder sb = new StringBuilder();

            foreach (var item in text)
            {

                sb.Append(item.ToString() + "\t");
            }

            return sb.ToString();
        }
        #endregion

    }





}